
#include <unistd.h>
#include <stdlib.h>
#include <pthread.h>
#include <signal.h>
#include <sys/sysinfo.h>
#include <algorithm>
#include "webcollector.h"
#include "webactivite.h"
#include "../core/operator.h"
#include "../core/webpageadapter.h"
#include <google/coredumper.h>

//#include <QProcess>
#include <QFileInfoList>
#include <QDir>
#include <QNetworkProxyFactory>

#include <QDebug>

static bool sg_sigexit = false;
// signal quit
void signal_quit(int no)
{
    Log4Qt::Logger *logger = Log4Qt::Logger::logger("SIGNAL");
#if 0

    WebSpider *s = WebCollector::instance()->getSpider(SPIDER_NAME);
    if (s != NULL) {
        QString log = "Spider Core : " +  s->protocol();
        logger->error(log);
    }

    qDebug() << "Running count : " << WebCollector::instance()->getRunningActiviteCount();
    qDebug() << "Running : " << WebCollector::instance()->getRunningUrls();
    qDebug() << "Wait count : " << WebCollector::instance()->getWaitActiviteCount();

    if (no == SIGSEGV || no == SIGABRT) {
        sg_sigexit = true;

        if (no == SIGSEGV)
            logger->error("Down by SIGSEGV");
        else
            logger->error("Down by SIGABRT");

        if (WebCollector::instance()->debug()) {
            int pid = getpid();
            QString coredumpPath = "/tmp/coredump-";
            coredumpPath.append(QString::number(pid));
            if( WriteCoreDump(coredumpPath.toLatin1().constData()) ==0 )
                logger->error("success: WriteCoreDump to coredump-pid file.\n");
            else {
                logger->error("failure: WriteCoreDump to coredump-pid file.\n");
                logger->error(coredumpPath);
            }
        }
    }
#else
    logger->info("Exit by signal!");
#endif
    _exit(0);
}

WebCollector* WebCollector::m_instance = NULL;

#define SCHEDUBCOUNT 2
#define ALLOWSTARTCOUNT 1

WebCollector::WebCollector()
    : QObject(0)
    , m_logger(Log4Qt::Logger::logger("WebCollector"))
    , m_maxActiviteCount(MAX_ACTIVITE_COUNT)
    , m_pid(getpid())
    , m_ppid(getppid())
    , m_isSleeping(false)
    , m_hasWorkedCount(0)
    , m_hasWorkedOutLimit(false)
    , m_dying(false)
    , m_hasDeleteLater(false)
    , m_allowStartCount(ALLOWSTARTCOUNT)
    , m_scheduBCount(SCHEDUBCOUNT)
    , m_isDebug(false)
    , m_isStatic(false)
    , m_exitWorkCount(EXIT_WORK_COUNT)
    , m_enableProxy(false)
{
    signal(SIGSEGV, signal_quit);
    signal(SIGABRT, signal_quit);

    m_timerId = startTimer(5);

    m_proxyUpdateTimer = new QTimer(this);
    connect(m_proxyUpdateTimer, SIGNAL(timeout()), this, SLOT(updateProxy()));
}

WebCollector::~WebCollector()
{
    if (m_timerId)
        killTimer(m_timerId);
}

void WebCollector::setProxy(QString proxyType,
        QString proxyAuthUser,QString proxyAuthPass,QString proxyHost,int proxyPort)
{
    m_proxyType = proxyType;
    m_proxyAuthUser = proxyAuthUser;
    m_proxyAuthPass = proxyAuthPass;
    m_proxyHost = proxyHost;
    m_proxyPort = proxyPort;
}

void WebCollector::enableProxy()
{
    m_enableProxy = true;
    if (m_proxyType != "none") {
        if (m_proxyHost.isEmpty()) {
            QNetworkProxyFactory::setUseSystemConfiguration(true);
        } else {
            QNetworkProxy::ProxyType networkProxyType = QNetworkProxy::HttpProxy;

            if (m_proxyType == "socks5") 
                networkProxyType = QNetworkProxy::Socks5Proxy;

            if(!m_proxyAuthUser.isEmpty() && !m_proxyAuthPass.isEmpty()) {
                QString logMessage = "Enable proxy " + m_proxyHost + ":" + QString::number(m_proxyPort) + " with user " + m_proxyAuthUser;
                m_logger->info(logMessage);
                QNetworkProxy proxy(networkProxyType, m_proxyHost, m_proxyPort, m_proxyAuthUser, m_proxyAuthPass);
                QNetworkProxy::setApplicationProxy(proxy);
            } else {
                QString logMessage = "Enable proxy " + m_proxyHost + ":" + QString::number(m_proxyPort);
                m_logger->info(logMessage);
                QNetworkProxy proxy(networkProxyType, m_proxyHost, m_proxyPort);
                QNetworkProxy::setApplicationProxy(proxy);
            }
        }
    }
}

void WebCollector::disableProxy()
{
    m_enableProxy = false;
    QNetworkProxy proxy;
    proxy.setType(QNetworkProxy::NoProxy);
    QNetworkProxy::setApplicationProxy(proxy);
}

WebCollector* WebCollector::instance()
{
    if (!m_instance)
        m_instance = new WebCollector();

    return m_instance;
}

void WebCollector::insert(WebSpider *spider)
{
    if (m_spiders.find(spider->name()) != m_spiders.end()) {
        QString logMessage = "The spider : " + spider->name() + "is exists!";
        m_logger->info(logMessage);
        return ;
    }

    m_spiders.insert(spider->name(), spider);
}

WebSpider* WebCollector::getSpider(QString key)
{
    if (m_spiders.find(key) != m_spiders.end())
        return m_spiders.find(key).value();

    return NULL;
}

static bool ActivitesSort(WebActivite* first, WebActivite* second)
{
    return (first->level() > second->level());
}

Operator* WebCollector::createActiviteWithCompletedWebPage(WebPage *page)
{
    WebSpider *spider = NULL;
    QMap<QString, WebSpider*>::iterator it;
    for (it = m_spiders.begin(); it != m_spiders.end(); it++) {
        if (!(*it)->name().isNull()) {
            QString logMessage = "Create webpage : " + page->url();
            m_logger->info(logMessage);
            spider = (*it);
            break;
        }
    }

    if (spider == NULL)
        return NULL;

    WebSpiderSyncRequest *req = page->rawRequest();
    // We stop wait time when request be handled.
    req->stopWaitTimer();
    WebSpiderAround *spiderAround = new WebSpiderAround(static_cast<WebSpiderRequest*>(req));
    WebActivite *activite = new WebActivite(spiderAround);
    m_startsActivites.push_back(activite);

    m_hasWorkedCount ++;

    WebSpider *s = getSpider(SPIDER_NAME);
    if (s)
        s->uploadState();

    return activite->startWithCompletedWebPage(page);
}

WebActivite* WebCollector::createActivite(WebSpiderRequest *req)
{
    WebSpiderAround *spiderAround = new WebSpiderAround(req);
    WebActivite *activite = new WebActivite(spiderAround);

    m_waitActivites.push_back(activite);
    m_waitActivites.sort(ActivitesSort);

    return activite;
}

bool WebCollector::newActivite(WebSpiderRequest *request)
{
    // worked count , for restart.
    // we will restart a new process and kill this process when activite has been create larger than limit times,
    // and they were all completed.
    m_hasWorkedCount ++;

    WebSpider *s = getSpider(SPIDER_NAME);
    if (s)
        s->uploadState();

    if (m_hasWorkedCount >= m_exitWorkCount)
        m_hasWorkedOutLimit = true;

    createActivite(request);
    
    return true;
}

void WebCollector::removeActiviteByOpt(Operator *opt)
{
    if (!opt)
        return ;

    for(int i = 0; i < m_startsActivites.size(); i ++) {
        WebActivite *act = m_startsActivites.at(i);
        if (opt == act->webAdapter()->opt()) {
            act->stop();
            m_startsActivites.removeOne(act);
            delete act;
            m_hasDeleteLater = true;
            return ;
        }
    }
}

double WebCollector::getSystemFreeRamPercent()
{
    struct sysinfo sysInfo;
    if (!sysinfo(&sysInfo)) {
        double percent = ((double)sysInfo.freeram/(double)sysInfo.totalram)*100;
        return percent;
    }

    return 0;
}

bool WebCollector::checkPPP0State()
{
    return true;
}

QList<QString> WebCollector::getRunningUrls()
{
    QList<QString> infos;
    for (int i = 0; i < m_startsActivites.size(); i ++) {
        QString info = QString::number(m_startsActivites.at(i)->loadingProcess());
        info.append("% --- ");
        info.append(m_startsActivites.at(i)->url());
        infos.push_back(info);
    }

    return infos;
}

void WebCollector::getPageInfos(ProcessRouter::CorePages *pages)
{
    pages->clear_pageinfos();
    for (int i = 0; i < m_startsActivites.size(); i ++) {
        ProcessRouter::PageInfo *info = pages->add_pageinfos();
        info->set_loadprocess(m_startsActivites.at(i)->loadingProcess());
        info->set_url(m_startsActivites.at(i)->url().toLatin1().constData());
        info->set_ref(QString::number(m_startsActivites.at(i)->ref()).toLatin1().constData());
        info->set_clientid(m_startsActivites.at(i)->clientID());
        info->set_requestts(m_startsActivites.at(i)->requestTs());
        info->set_startts(m_startsActivites.at(i)->startTs());
    }

    std::list<WebActivite*>::iterator it;
    for(it = m_waitActivites.begin(); it != m_waitActivites.end(); it++) {
        ProcessRouter::PageInfo *info = pages->add_pageinfos();
        info->set_loadprocess((*it)->loadingProcess());
        info->set_url((*it)->url().toLatin1().constData());
        info->set_ref(QString::number((*it)->ref()).toLatin1().constData());
        info->set_clientid((*it)->clientID());
        info->set_requestts((*it)->requestTs());
        info->set_startts((*it)->startTs());
    }
}

int WebCollector::getRunningActiviteCount()
{
    return m_startsActivites.size();
}

int WebCollector::getWaitActiviteCount()
{
    return m_waitActivites.size();
}

int WebCollector::getHasWorkedCount()
{
    return m_hasWorkedCount;
}

// 不再接收页面请求
void WebCollector::killSelf()
{
    if (m_dying)
        return;

    QMap<QString, WebSpider*>::iterator it;
    for (it = m_spiders.begin(); it != m_spiders.end(); it ++) {
        (*it)->destroy();
    }
    m_dying = true;
}

// 退出
void WebCollector::exitSelf()
{
    if (!sg_sigexit) {
        m_logger->info("Exit by command kill!");
        _exit(0);
    }
}

void WebCollector::scheduLoop(QTimerEvent *event)
{
    // System memory control. SYSTEM_MEMORY_LIMIT
    double systemFreeMemoryPercent = getSystemFreeRamPercent();

    QMap<QString, WebSpider*>::iterator spiderIt;
    for (spiderIt = m_spiders.begin(); spiderIt != m_spiders.end(); spiderIt ++)
        (*spiderIt)->taskLoop();

    // WebKit of system memory will be used more and more, we present the solution is to restart when the task amount exceeds the limit.
    if (!m_dying && m_isSleeping && m_hasWorkedOutLimit)
        killSelf();

    // Change to sleeping when activite is empty, wakeup when not empty.
    if (!m_isSleeping) {
        if (m_startsActivites.isEmpty() && m_waitActivites.empty()) {
            QMap<QString, WebSpider*>::iterator it;
            m_isSleeping = true;

            m_logger->info("Sleeping");
            startUpdateProxyTimer();
        } // else not empty, running.
    } else {
        if (!m_startsActivites.isEmpty() || !m_waitActivites.empty()) {
            // wakeup
            stopUpdateProxyTimer();
            m_isSleeping = false;
        }
    }

    // Exit when m_dying is true and all activites stop.
    if (m_dying && m_isSleeping) {
        m_logger->info("Spider webkit core exit!");
        QMap<QString, WebSpider*>::iterator it;
        // Send message to maintain first.
        for (it = m_spiders.begin(); it != m_spiders.end(); it ++)
            (*it)->sendDestroyMessage();
    }

    if (m_isSleeping) {
        // exit when webkit is free and worked count > exit count
        if (m_hasWorkedCount > m_exitWorkCount) {
            m_logger->info("Work Free Exit, Spider webkit core exit!");
            QMap<QString, WebSpider*>::iterator it;
            for (it = m_spiders.begin(); it != m_spiders.end(); it ++)
                (*it)->sendDestroyMessage();
        }
        return;
    }

    if (m_waitActivites.empty()) {
        return ;
    }

    scheduActivites();

    for (spiderIt = m_spiders.begin(); spiderIt != m_spiders.end(); spiderIt ++) {
        QString logMessage = "Schedu | " + (*spiderIt)->protocol()
                           + " | running : " + QString::number(m_startsActivites.size())
                           + " wait : " + QString::number(m_waitActivites.size())
                           + " memory : " + QString::number(systemFreeMemoryPercent);
        m_logger->info(logMessage);
    }

    // 减少max activite数量以便快速退出
    if (m_hasWorkedCount > HAS_WORKED_LIMIT_PREPARE) {
        m_maxActiviteCount = WAR_ACTIVITE_COUNT;
        return;
    }

    // Running activite count need change by free ram memory.
    if (systemFreeMemoryPercent < MIN_MEMORY_LIMIT) {
        m_maxActiviteCount = MID_ACTIVITE_COUNT;
    } else if (systemFreeMemoryPercent < LIMIT_MEMORY_LIMIT) {
        m_maxActiviteCount = WAR_ACTIVITE_COUNT;
    } else
        m_maxActiviteCount = MAX_ACTIVITE_COUNT;

}

void WebCollector::scheduActivites()
{
    if (m_hasDeleteLater) {
        m_hasDeleteLater = false;
        return;
    }

    if (m_scheduBCount) {
        m_scheduBCount --;
        return;
    }

    m_scheduBCount = SCHEDUBCOUNT;

    if (!checkPPP0State()) {
        return ;
    }

    int allowStart = m_allowStartCount;

    for (int i = 0; i < m_maxActiviteCount - m_startsActivites.size(); i ++) {
        if (m_waitActivites.empty())
            break;

        if (!allowStart)
            break;
        allowStart --;

        WebActivite *activite = m_waitActivites.front();
        m_startsActivites.push_back(activite);
        m_waitActivites.remove(activite);
        activite->start();
    }
}

void WebCollector::updateProxy()
{
    WebSpider *s = getSpider(SPIDER_NAME);
    if (s)
        s->updateProxy();
}

void WebCollector::startUpdateProxyTimer()
{
    m_proxyUpdateTimer->start(UPDATE_PROXY_TIME);
}

void WebCollector::stopUpdateProxyTimer()
{
    if (m_proxyUpdateTimer->isActive())
        m_proxyUpdateTimer->stop();
}

void WebCollector::timerEvent(QTimerEvent *event)
{
    scheduLoop(event);
}

void WebCollector::setDebug()
{
    m_isDebug = true;
}

bool WebCollector::debug()
{
    return m_isDebug;
}

void WebCollector::setStatic()
{
    m_isStatic = true;
}

bool WebCollector::isStatic()
{
    return m_isStatic;
}

void WebCollector::setExitWorkCount(int count)
{
    m_exitWorkCount = count;
}

int WebCollector::exitWorkCount()
{
    return m_exitWorkCount;
}

